{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "GPU_id = 3\n",
    "os.environ['CUDA_VISIBLE_DEVICES'] = str(GPU_id)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "\n",
    "import cudf as gd\n",
    "import cupy as cp\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import os\n",
    "import time\n",
    "import nvstrings\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Global"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "GPU_RUN_TIME = {}\n",
    "CPU_RUN_TIME = {}\n",
    "STEPS = []"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def on_gpu(words,func,arg=None,dtype=np.int32):\n",
    "    res = cp.array(words.size(), dtype=dtype)\n",
    "    if arg is None:\n",
    "        cmd = 'words.%s(devptr=res.device_ctypes_pointer.value)'%(func)\n",
    "    else:\n",
    "        cmd = 'words.%s(arg,devptr=res.device_ctypes_pointer.value)'%(func)\n",
    "    eval(cmd)\n",
    "    return res\n",
    "\n",
    "def count_items(data,cols):\n",
    "    dg = data.groupby(cols+['item_id'],\n",
    "            as_index=False).agg({'step':['count']})\n",
    "    if len(cols) == 0:\n",
    "        tag = 'global'\n",
    "    else:\n",
    "        tag = '_'.join(cols)\n",
    "    dg.columns = cols + ['item_id', 'count_item_%s'%tag]\n",
    "\n",
    "    if len(cols):\n",
    "        df = data.groupby(cols,\n",
    "            as_index=False).agg({'step':['count']})\n",
    "        df.columns = cols + ['count_item_%s_all'%tag]\n",
    "    \n",
    "        dg = dg.merge(df,on=cols,how='left')\n",
    "        dg['count_item_%s_norm'%tag] = dg['count_item_%s'%tag] / dg['count_item_%s_all'%tag]\n",
    "    \n",
    "        dg = dg.drop('count_item_%s_all'%tag,axis=1)\n",
    "        del df\n",
    "    return dg"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Read csv data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "if os.path.exists('cache')==False:\n",
    "    os.mkdir('cache')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "path = '/datasets/trivago/data/'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### cudf read csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "combined (19715327, 6)\n",
      "CPU times: user 2.69 s, sys: 1.69 s, total: 4.38 s\n",
      "Wall time: 6.11 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "cols = ['city','user_id', 'session_id', 'step', 'action_type', 'reference']\n",
    "\n",
    "train_gd = gd.read_csv('%s/train.csv'%path,usecols=cols)\n",
    "test_gd = gd.read_csv('%s/test.csv'%path,usecols=cols)\n",
    "data_gd = gd.concat([train_gd,test_gd])\n",
    "\n",
    "\n",
    "data_gd = data_gd[cols]\n",
    "cols = ['city','user_id', 'session_id', 'step', 'action_type', 'item_id']\n",
    "data_gd.columns = cols\n",
    "\n",
    "print('combined',data_gd.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 428 ms, sys: 960 ms, total: 1.39 s\n",
      "Wall time: 2.11 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "cols = ['user_id','session_id','item_id','city','device']\n",
    "data_pair_gd = gd.read_parquet('../cache/data_pair.parquet')[cols]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "del train_gd,test_gd"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Only keep interaction rows"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "step = 'contrain string'\n",
    "STEPS.append(step)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### cudf find string within string"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(268470,)\n",
      "CPU times: user 2.22 s, sys: 824 ms, total: 3.04 s\n",
      "Wall time: 4.05 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "start = time.time()\n",
    "\n",
    "data_gd['is_interaction'] = on_gpu(data_gd['action_type'].data,'contains',\n",
    "                                          arg='interaction',dtype=np.bool)\n",
    "data_interaction_gd = data_gd[data_gd['is_interaction']]\n",
    "data_interaction_gd['item_id'] = data_interaction_gd['item_id'].astype(int)\n",
    "print(data_interaction_gd['item_id'].unique().shape)\n",
    "GPU_RUN_TIME[step] = time.time() - start"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### count items with/without other columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "step = 'count items'\n",
    "STEPS.append(step)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 1.22 s, sys: 1.46 s, total: 2.68 s\n",
      "Wall time: 5.86 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "start = time.time()\n",
    "count_user_session_gd = count_items(data_interaction_gd,cols=['user_id','session_id'])\n",
    "count_user_session_city_gd = count_items(data_interaction_gd,cols=['user_id','session_id','city'])\n",
    "count_user_city_gd = count_items(data_interaction_gd,cols=['user_id','city'])\n",
    "count_city_gd = count_items(data_interaction_gd,cols=['city'])\n",
    "count_global_gd = count_items(data_interaction_gd,cols=[])\n",
    "GPU_RUN_TIME[step] = time.time() - start"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Merge the encoding to pair"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "step = 'merge'\n",
    "STEPS.append(step)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### cudf merge"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 5.36 s, sys: 4.09 s, total: 9.45 s\n",
      "Wall time: 12.3 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "start = time.time()\n",
    "data_pair_gd = data_pair_gd.merge(count_user_session_gd,on=['user_id','session_id','item_id'],how='left')\n",
    "data_pair_gd = data_pair_gd.merge(count_user_session_city_gd,on=['user_id','session_id','city','item_id'],how='left')\n",
    "data_pair_gd = data_pair_gd.merge(count_user_city_gd,on=['user_id','city','item_id'],how='left')\n",
    "data_pair_gd = data_pair_gd.merge(count_city_gd,on=['city','item_id'],how='left')\n",
    "GPU_RUN_TIME[step] = time.time() - start"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Visualize the timing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>step</th>\n",
       "      <th>GPU</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>contrain string</td>\n",
       "      <td>4.048962</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>count items</td>\n",
       "      <td>5.857505</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>merge</td>\n",
       "      <td>12.288357</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Overall</td>\n",
       "      <td>22.194824</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              step        GPU\n",
       "0  contrain string   4.048962\n",
       "1      count items   5.857505\n",
       "2            merge  12.288357\n",
       "3          Overall  22.194824"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "GPU_RUN_TIME['Overall'] = sum([GPU_RUN_TIME[i] for i in STEPS])\n",
    "STEPS.append('Overall')\n",
    "\n",
    "timing = pd.DataFrame()\n",
    "timing['step'] = STEPS\n",
    "timing['GPU'] = [GPU_RUN_TIME[i] for i in STEPS]\n",
    "timing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
